{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "e796976a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9d3ad16d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from dynamic_quant_ops import tensor_quant_gelu, tensor_quant_scale\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3f334006",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = torch.Tensor([-3, -1, 1, 3, 5])\n",
    "gelu_gt = torch.nn.functional.gelu(inputs)\n",
    "gelu_test = tensor_quant_gelu(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "453e9a96",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Line magic function `%%plt.plot(inputs,` not found.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6zklEQVR4nO3deViU5eLG8e+AsqiAqAiiuOCa+26apSbuuVVqZuZRT6lpaWalZcd2zOpki6llaYumpaEncz2uae6K+4JbmojiwqoMMPP+/qD85UlLcIZ3Zrg/1zXXFcMwc78Bzs3zPO/zWgzDMBARERFxAC+zA4iIiIjnULEQERERh1GxEBEREYdRsRARERGHUbEQERERh1GxEBEREYdRsRARERGHUbEQERERhymU3y9ot9uJj48nICAAi8WS3y8vIiIieWAYBqmpqYSHh+PldfNxiXwvFvHx8UREROT3y4qIiIgDnD59mnLlyt308/leLAICAoCcYIGBgfn98iIiIpIHKSkpREREXHsfv5l8Lxa/T38EBgaqWIiIiLiZv1vGoMWbIiIi4jAqFiIiIuIwKhYiIiLiMCoWIiIi4jAqFiIiIuIwKhYiIiLiMCoWIiIi4jAqFiIiIuIwKhYiIiLiMLkqFi+//DIWi+W6W40aNZyVTURERNxMrrf0rlWrFv/973///wkK5fuu4CIiIuKict0KChUqRFhYmDOyiIiIiJvL9RqLuLg4wsPDiYyMpF+/fpw6deovH2+1WklJSbnuJiIiIo73xhvw0ktgGOZlyFWxaNasGbNmzWLZsmVMnTqVEydOcPfdd5OamnrTr4mOjiYoKOjaLSIi4rZDi4iIyPXmzIHx4+H11+EPKxbyncUw8t5rkpKSqFChAv/+978ZPHjwDR9jtVqxWq3XPv79eu7Jycm6bLqIiIgDrF8P7dpBZiaMGQNvv+3410hJSSEoKOhv379va+Vl8eLFqVatGkePHr3pY3x9ffH19b2dlxEREZGbOL70MMndX6BI1gy6PhDMW2+Zm+e29rFIS0vj2LFjlClTxlF5RERE5BZdOHAe726d6Zr1PbNLjeSrr8DL5B2qcvXyY8aMYd26dZw8eZKff/6Znj174u3tTd++fZ2VT0RERG7g6qWrJDTrRoXs4/xSKJIma9/B39/sVLmcCvn111/p27cvFy9eJCQkhJYtW7J582ZCQkKclU9ERET+hz3bzu66/bkzbQuXLcHY/rOEkFqlzY4F5LJYzJ0711k5RERE5Bb91OJ5Wp1ZgBUffpm8kPqdqpsd6RpdK0RERMSNrH/kE1pteweAbcNmUv+pe0xOdD0VCxERETexbBkM/6YlJ6nA6ntfp+XHD5sd6U90oQ8RERE3sHs39O4NqfaaTOoby5Svg8yOdEMasRAREXFxCdtO81rUOlJToU0bmDyrOBYvi9mxbkjFQkRExIWl/ppM6j2dmXOhHU+UXcSCBeDjY3aqm1OxEBERcVHZV7OIa9CLqhn7uOxVkufnNiA42OxUf03FQkRExAUZdoPNjZ6g4YWVpFGUxJk/Ur5lebNj/S0VCxERERe04b6JtDw4Axte7H1hLrUfbWh2pFuiYiEiIuJitjz9DXcvfQGAdfd/QPM37jM50a1TsRAREXEhW7bAwQ//C8CqeqNpM3+4yYlyR/tYiIiIuIjjx6FrV0i0zeB8/ShGb+mDxTXPKr0pjViIiIi4gMu/pHBfZzuJidCggYUnfupLIR/3e5t2v8QiIiIeJjP5KmfqduTVw72pUvYqixdDsWJmp8obTYWIiIiYyLDZia37KE1TNlGW4iye9ivh4VXNjpVnGrEQEREx0cZ7xtH01HwyKczRSTFUv899SwWoWIiIiJhm08DptPx5EgAbB31Ok2dbmxvIAVQsRERETLDnraU0mZVzKunKlq/Q5rNHTE7kGCoWIiIi+ezQjnTKjBtAIWysqziAtmtfMjuSw6hYiIiI5KNz56DTg0XpbixkVcneNNv9CV7ebrZZxV/QWSEiIiL55MoV6NYNTp4E78otqLe5BX6BZqdyLI1YiIiI5AO7NYtNNQdh3RpLiRKwZAmUKmV2KsfTiIWIiIizGQZbmwyn7S8zWcYSjs87TrVqRcxO5RQasRAREXGyTT0ncefeT7Fj4cgzn9AiyjNLBahYiIiIONXOsd/SfNFYAFZ2eZ973ulmciLnUrEQERFxkiMzN1LzrUcBWFlzJO1/eNLkRM6nYiEiIuIE8T8do9Tg7vhhZWNId1rveNftLoGeFyoWIiIiDpaSAg8MDWGr0Zi9/k2oHTubwn7eZsfKFzorRERExIGysqB3b9h8IJDHQn/g5+WpBIUXNTtWvtGIhYiIiIMYNjufdFnE8uUGRYpAzOLCRNQrYXasfKURCxEREQfZHPUiw9dOxM6TVPjmAxo3NjtR/lOxEBERcYDtQz6l+dqJAFTv14j2nn1W6U1pKkREROQ2HXhvOfU/GQbA0qYTaP/1AJMTmUfFQkRE5DacWryHiGd6UQgba8r1p/3GCWZHMpWKhYiISB5d3neGwj27EGCksj2gNU33zMC7UAHYrOIvqFiIiIjkgdUKk3ttpHR2PEcL1SBi6/cUDfYxO5bptHhTREQklwwDBg2COYd6s88/gOiYGoTWCDY7lktQsRAREckNw+D1senMmVOMQoXgiR86Ua2t2aFch6ZCREREcmFrn3fpNakxlTjOJ59AW5WK66hYiIiI3KI9L31H0++epQaHmdxhGQMHmp3I9ahYiIiI3IJjX2+i2uv9AVhe7Um6Ln3C5ESuScVCRETkb5zfdIziA7rlXAK9RFda73qvQFwCPS9ULERERP5C+qmLXG3TmZL2C+z1bUSNXd/gW6RgXAI9L1QsREREbsJmg20tRlLBeoTTXuUptmYxJcsXnEug54WKhYiIyE088wz0OfMuKy3tufDFEio1DzM7ksvTPhYiIiI38MEH8P77AKEkzVtOu15mJ3IPGrEQERH5H7ue+pydI78AYOJE6KVSccs0YiEiIvIHR6asoM6HjzMLG3W6lGf0c23MjuRWNGIhIiLym/jleynz5IMUwsaqMo/w1PetdVppLqlYiIiIACmH4rF0/e0S6EXvocnuGRT2UavILRULEREp8LIup3Gu6X2UyTrN0ULVKbMphsAQX7NjuSUVCxERKdCMbBsH6j1E1dRdJBKC9fsllK1TwuxYbkvFQkRECrTot7xYeLoRV/DnyDv/oVbXSLMjuTWdFSIiIgXWN9/Ai+MtwCtUev2fPPpMhNmR3J5GLEREpEDa/fFGhg24AsDo0fDoiyoVjnBbxWLixIlYLBZGjRrloDgiIiLOd+q7LVQbHsXyrDY80uUyb79tdiLPkedisW3bNqZPn07dunUdmUdERMSpLm47TtGHuuJPBllBIUyfE4CXxu8dJk//K9PS0ujXrx+ffvopwcHBjs4kIiLiFBnxl0hr1ZmS9kT2+TSg2s65FAnUckNHylOxGD58OF26dCEqKsrReURERJzCftXK8fo9qXD1ML9aIvBdsZjSkcXMjuVxcl3T5s6dy86dO9m2bdstPd5qtWK1Wq99nJKSktuXFBERuT2Gwe4mg2mQuJ5kAon/9Eeatgo3O5VHytWIxenTpxk5ciSzZ8/Gz8/vlr4mOjqaoKCga7eICK26FRGR/DXnrdOU3b+CLAqx9bkFNB1cx+xIHstiGIZxqw9euHAhPXv2xNvb+9p9NpsNi8WCl5cXVqv1us/BjUcsIiIiSE5OJjAw0AGHICIicnPLl0OXLlDedpyJD+6g93e6BnpepKSkEBQU9Lfv37maCmnbti179+697r6BAwdSo0YNnn/++T+VCgBfX198fbXfuoiI5L+92zLo1csPmw1a9o+k1xfaVdPZclUsAgICqF279nX3FS1alJIlS/7pfhERETOdX72PkHYduNv+CVdad2HGDHQJ9HygM3dFRMTjpB89i61jF8Ls8fyryDt8v8DAx8fsVAXDbZ+8u3btWgfEEBERcYzspDTONrqPKlmnOOpdjdD18wkuoaGK/KIRCxER8RhGto2DDR6mSspOEilF2rwlVGxU0uxYBYqKhYiIeAbDILbNKOqc/IEMfNn3xn+o/0Bls1MVOCoWIiLiEbaMjaHBho8AWNH/a9q80NzkRAWTNkgXERG3t3UrRH3QjUkMJfTOSHp+8aDZkQosFQsREXFrJ05A166QllGIxZ0+ZtEinVZqJk2FiIiI20qOPcHqxs9y6XwW9evD3HkWChVWqzCTRixERMQtZZ67zOW7ujD4ykHsRbPpvPg9AgLMTiUasRAREbdjWDM5Vu9+Kl45yBlLWVp8P4ayZc1OJaBiISIi7sYw2NPsn9xxbi0pBHBiyhJqtVercBUqFiIi4lZ2P/Aq9XZ/RTbebHjqO1oOq2t2JPkDFQsREXEbB8d+Qb2YlwFY1GEqnd/vYG4g+RMVCxERcQuHDsG/PipNKsWIqTGWnkseMzuS3IDOChEREZd3/jx07gwn0jtBg1i+/KkSXvrT2CXp2yIiIi7t6okEhrU/xokTEBkJHy+vjH9RvX25Kn1nRETEZdlT0znTqCvTdt9J24CtLFkCISFmp5K/omIhIiKuyWbjQMN+VLm8HQsGb04rQfXqZoeSv6NiISIiLmlPu2eofXQRGfiy81+LaPpwFbMjyS1QsRAREZdzYMj71F3zPgA/9v6S9q/cZXIiuVUqFiIi4lKOv7eIGp88DcB3jSZy/9zeJieS3FCxEBERl/HraYP4sR/ghcHi8MfpvvE5XQLdzahYiIiIS0hJgS73WWiXuZj3Sr9Jy91T8PFVq3A3KhYiImK67Ixs+vSBPXsgKNSfnlvGUbyU9nB0RyoWIiJiKsOaSVyVjrRY9hL+fgY//AAVK5qdSvJKxUJERMxjGOxrOYQ7zqxiFJNZNPkETZqYHUpuh4qFiIiYZt9Dr1Nn+yyy8WbV49/Sbkik2ZHkNqlYiIiIKeImfE3tb/8FwPw2U+gxvZPJicQRVCxERCTfnZm9lgqvDgJgQeXn6LVyiMmJxFFULEREJF9dPnaJoo/ejw9ZrCzeiw47o/H2NjuVOIqKhYiI5BurFboPLMGT9vf52bc1tXd8QbFAvRV5En03RUQkXxgGDBoEP/0E/wnsT+C21ZSJ9Dc7ljiYioWIiDifzcb6u19k1ZwEChWC+fOhdh3tqumJVCxERMTp9nd+llYb32QtrZn+URbt2pmdSJxFxUJERJzq8JMfUWvFewDEdn+ZQUMKm5xInEnFQkREnObUlB+o8tFIAL6pG03v7x8yOZE4m4qFiIg4xYXlOyj15EN4Y+c/of+k5+bn8dK7jsfTt1hERBzuyqFT0PU+ihhX+Mm/PS12fYyfvxZrFgQqFiIi4lA2G4x6yk5iVnH2e9chfON3lCqjdRUFhYqFiIg41Jgx8OnKirTx+Zkr85dSuUGg2ZEkH6lYiIiIYxgG3zy3i8mTcz58/8tgmvQoa2okyX8qFiIi4hCHHn2T3m835gmm8Oab0KeP2YnEDIXMDiAiIu7v+BtzqPH1eABatLDw8FiTA4lpNGIhIiK35dx36yk7fiAA8ys8Q++1T2DRCSAFloqFiIjkWer2w/j17YEvmawIfIB2uyZRWCeAFGgqFiIikidZ8Ymk3tOZINtlthe+kzu2fUVQsN5WCjr9BIiISK4ZBiy4fzbhV49z3BKJz5JFRFTTJdBFizdFRCQPJk6EF7aMZJPFix4fd6BNVGmzI4mLULEQEZFcmTvHzgsveAEWqn7wFG2Gmp1IXImmQkRE5JYdfWYqpR7pSCDJjBoFI0aYnUhcjUYsRETklpz5dAmV/j2CKth5q943PPaOhirkzzRiISIif+vy6l0UH9I75xLopQbx6MYheHubnUpckYqFiIj8pYy409g6dqGokc5PflE02zWNIkW1A5bcmIqFiIjclP1yMuebdKZU1lkOeNUiZO18QstpByy5ORULERG5qQPNB1M+eR9nCePy10uo0SzI7Eji4lQsRETkhmbMgH6HX+IQ1dn+8mLu6lve7EjiBnJVLKZOnUrdunUJDAwkMDCQ5s2bs3TpUmdlExERk6xYAUOHwh7q8c2L++k6oZHZkcRN5KpYlCtXjokTJ7Jjxw62b9/OvffeS/fu3dm/f7+z8omISD775d35vNNjAzYbPPIIvPyaTv+QW2cxDMO4nScoUaIEb7/9NoMHD76lx6ekpBAUFERycjKBgYG389IiIuJgFxZuIKBnWwwsjGjwM1M2NcTX1+xU4gpu9f07z2ssbDYbc+fOJT09nebNm+f1aURExEWk7zpCoQe740sm64t1ZtLyeioVkmu53nlz7969NG/enIyMDIoVK0ZMTAw1a9a86eOtVitWq/XaxykpKXlLKiIiTmO/nEzS3V0pa7vEjkJNqbLpa0qEaApEci/XIxbVq1cnNjaWLVu2MGzYMAYMGMCBAwdu+vjo6GiCgoKu3SIiIm4rsIiIOJjdTtydj1A2/QiniMBY+B8iaxcxO5W4qdteYxEVFUXlypWZPn36DT9/oxGLiIgIrbEQEXERhx6aQI15r5KBLyte2kC3VxubHUlc0K2usbjti5DZ7fbrisP/8vX1xVeTdCIiLungfjuH5++nBvBd1Cf0V6mQ25SrYjFu3Dg6depE+fLlSU1NZc6cOaxdu5bly5c7K5+IiDhJUhJ07+nFUdu3jK7zX6KXtDc7kniAXBWL8+fP8+ijj3L27FmCgoKoW7cuy5cvp127ds7KJyIiTmBLu0q/h/2Ii7MQEeHFc/9tT2FdAkQcIFfF4rPPPnNWDhERyS92O3ENetPvaACbfGcQE1OE0qXNDiWe4rbXWIiIiHs50Ptlah5dTAX8CHzpeRo1qmd2JPEgugiZiEgB8sv7MdRc8BoAMR0/4b4XVSrEsVQsREQKiKSfD1Dy6UcBWBAxkt4/9Dc5kXgiFQsRkQLAdjGJtHY9KGakscmvNa22vE0hTYaLE6hYiIgUAAebD6TclThOWcoTsORbSpXRKSDiHCoWIiIebu5ceCLuaU5SgUNvxlC7TYjZkcSDaSBMRMSDxcbCoEFwlXv4dMwR3hjrY3Yk8XAasRAR8VCXNx7guS77uXoVOnSAVyeqVIjzqViIiHig7MTLpLfrzoL4O+kbvo45c8BbV0GXfKBiISLiaWw24pr2o9zVo1y2lOClb2pSooTZoaSgULEQEfEwe++fwB0nl3IVP45OiuGOe7RYU/KPioWIiAc5Oul76vznDQAWd5/BvWMampxIChoVCxERD3Fx/X7KjM3ZWXNhpad54Pt+JieSgkjFQkTEA2Rlwebe/6aokc4m/3tpvXUSXvoXXkygHzsREQ8wejT0ODeNN31epuR/51G8lLYpEnPoJ09ExM19/jl89BFAYWp/N4FqLcxOJAWZRixERNzYkegFXH3sKQqRxSuvQLduZieSgk4jFiIiburC2n2UfXEAw410Ctetzj/HDzc7kohGLERE3FHmuctkdOqRs1izSFseWjNEizXFJejHUETE3dhsHGnyMOUyjvGLpSIhq+cRWEID0OIaVCxERNzMrvteovbpZVzBn1MfLKRKs5JmRxK5RsVCRMSNHHp9Pg2WRQOwos/n3D2insmJRK6nYiEi4ibOnIE33vUlhQD+U20M3b95yOxIIn+iSTkRETeQkQH33w9bk7pyoVos320pj8VidiqRP9OIhYiIizOybbww8Cxbt0JwMHy0JJJixfV3obgmFQsRERe3s/OLjJ1bj9aWdcydC5Urm51I5OZUeUVEXNiBl7+l0cq3ABj98Fnatzc5kMjf0IiFiIiLOrt8DxVeGQjA4hpjuO8rLdYU16diISLigq6euYStWw+KcoXNxaK4d0u0FmuKW1CxEBFxMUa2jbgmfSmXeYKTXpUIXz+XIoGauRb3oGIhIuJi1vf6kLpnV3AFf85Pi6F8A+2sKe5DFVhExIWsWgU9/vM4U9lOyUfvo9Nj2llT3IuKhYiIizh5Evr0gTR7EVY++jWzZpmdSCT3NBUiIuICrpy+yHd3TebSRTuNGsG0aWixprgljViIiJjMyMrmaJO+PHtuJcX9j9Mx5gP8/c1OJZI3GrEQETHZtnYvUPfcStIpQoMpjxERYXYikbxTsRARMdHucXNpuu5tADYMmknjgXVMTiRye1QsRERMcnrxbqpOHATAkjrP035Gb5MTidw+FQsREROk/XIRy/09KcJVNgV14N5Nb2ixpngEFQsRkXxmGPB+v62EZJ3hpHckFTbMwa+ot9mxRBxCxUJEJJ9FR8P4jZ2413s9STMXEl67hNmRRBxGp5uKiOSjH3+wM358zt90/5jajPr9TQ4k4mAasRARySe/LIolskcd6hm7GDIEHnvM7EQijqdiISKSD1JPXsS7V0/usB9gconX+OADsxOJOIeKhYiIk9kzsznepA/lsk5ywrsyNTZ+ho+P2alEnEPFQkTEyTa3Hku9C6tIoyhpXy0ktEaw2ZFEnEbFQkTEibaPnkOLTe8CsPWJWdTpW9vkRCLOpWIhIuIkx2N2U/O9fwKwotE47p3yoMmJRJxPp5uKiDhBUhL0HlOe12hFcLCFNhteMzuSSL5QsRARcTCbDfr1gx3HgxlWbjHb1l+lsJ921pSCQVMhIiIONmPwzyxZYuDnBwsWehNSqZjZkUTyjUYsREQcaOtTXzPki/548U/8P/mERo10ZTEpWFQsREQc5Oi3O6nzYc52mpWahBDVX6VCCh5NhYiIOMDluAv49+uJPxlsKdmJ1j9psaYUTCoWIiK3yWbN5uSdfSibfYoThapQZfNsCvlqsaYUTLkqFtHR0TRp0oSAgABKly5Njx49OHz4sLOyiYi4hZ9bPkeDS6tJpRjWuQspWUU7a0rBlatisW7dOoYPH87mzZtZuXIlWVlZtG/fnvT0dGflExFxaUvePUiL7e8DsHv0F9R4oJbJiUTMZTEMw8jrFycmJlK6dGnWrVvHPffcc0tfk5KSQlBQEMnJyQQGBub1pUVETBcbCy1aQMurK3i61U46rR1rdiQRp7nV9+/bOiskOTkZgBIlStz0MVarFavVel0wERF3d+EC9OwJV6+CV4f2tP+xvdmRRFxCnhdv2u12Ro0axV133UXt2je/qE50dDRBQUHXbhEREXl9SRERl5B9NYtN9YbiffIolSvDnDngrbWaIsBtTIUMGzaMpUuXsmHDBsqVK3fTx91oxCIiIkJTISLittY3HMU9u97ntCWClO1x1Groa3YkEadz6lTIiBEjWLx4MevXr//LUgHg6+uLr69+6UTEM2wY+hX37MpZrHnm2fe5U6VC5Dq5mgoxDIMRI0YQExPD6tWrqVSpkrNyiYi4nINf76DR9McBWNtyPHe+1dPkRCKuJ1cjFsOHD2fOnDksWrSIgIAAEhISAAgKCsLf398pAUVEXEHigUQC/nE//mSwtXQX7lnzitmRRFxSrtZYWCw33vd+5syZ/OMf/7il59DppiLibrKuZLEvvD0NktdyonBVSsRtJahCcbNjieQrp6yxuI0tL0RE3Nb4p1LokZxBKsWwL1ioUiHyF3StEBGRv/D55zDps5K0Zi0731lD5a41zY4k4tJULEREbmL76hSGDcv57xdf8aXVM43NDSTiBlQsRERu4Py+85RpX5uXM8dxf3cb48ebnUjEPahYiIj8j8z0LM7c1YuyttM85PM9Mz9Kx0v/WorcEv2qiIj8jw13PkODlPWkEIBl4UICy+kMNpFbpWIhIvIHawbM4t59HwJwZPxXVOx0h8mJRNyLioWIyG92z9hG8y+HAvDTvRNo/Fp3kxOJuB8VCxERID4unVJDHsAPK9vKdKPlin+ZHUnELalYiEiBl5EBPR8pyhj7Wxzwa8gd27/C4q1/HkXyQr85IlKgGQYMHw5bt8Ly4L747tlGsXAt1hTJKxULESnQfnziR5Z8fhYvL5g7FypX1T+LIrcjV9cKERHxJDunbyNq2gNspyQ/jv2Z9u0rmB1JxO2pmotIgXRm5znCnuiJH1bOlm3MY69GmB1JxCOoWIhIgXM1OZNzrXoRbj/DCd/q1NRiTRGH0W+SiBQohgEbmo6mYdpPpFoC8PlxIUXCtFhTxFFULESkQFn58EzaHZkCwIlXv6Zs2xomJxLxLCoWIlJgrF5po9TcnO26N3d8mbrju5mcSMTz6KwQESkQTp6E3n29sbKOTxtOo8/iZ8yOJOKRNGIhIh7vyhXo0QMuXoTqjQLovuFZLdYUcRL9ZomIRzMMWF//KaJ2v0NIKYOYGPD3NzuViOfSVIiIeLTlvT+jY9yHdAQefqM1ERGNzY4k4tE0YiEiHmvLB1toM/8JALZ2eYWGj6tUiDibioWIeKSTmxOIGHU/vmSys3wPmiwab3YkkQJBxUJEPE7apUwutX2QcCOeE353UHP7l1qsKZJP9JsmIh7FMGBDk1E0vLKRZEsQRVYsxC8kwOxYIgWGioWIeJToaPjx+B1Y8SF+0mxC765mdiSRAkVnhYiIx/jxRxg/HgyepPlbPXl4TDmzI4kUOBqxEBGPcGzTeZ54OAnDgCFD4OHnVCpEzKBiISJuL+VCJklt72dlSlP6NjjEBx+YnUik4FKxEBG3ZrfDxiYjaXR1I6GW87z/kTc+PmanEim4VCxExK0t7jGDTienYcdCwrtzCGlR1exIIgWaioWIuK31kzbT4YfhAOzq+RrVn+5sciIRUbEQEbcUt/4sVcfm7KwZG3k/jRa8YHYkEUHFQkTcUFISxHUZRRnjLCeK1KTWtllgsZgdS0TQPhYi4mZsNujXD7alfchX/lYar3qbwiW0s6aIq1CxEBG3MmECLFkCfn6lKfXTQko2MjuRiPyRpkJExG2sfmMTp9/4AoBPP4VGKhUiLkcjFiLiFg6tjueOlx7gC87Ssb1B30f+YXYkEbkBjViIiMu7dNbKlc4P5izWLFqLXnMfMDuSiNyEioWIuDSbDTY1fYqG1k0kexUnaPVCCgVrsaaIq1KxEBGXtrDzJ3T59RPsWLj44TeUaFrF7Egi8hdULETEZa189We6rhgBwL6H3iDyiY4mJxKRv6NiISIuKTYWVr6+BR+y2FP9QerOGWt2JBG5BTorRERczoUL0LMnnMx6msIN7+DV1S21s6aIm1CxEBGXkp0N/XplcvKkD5UrwzMrO+IdZHYqEblVmgoREZcS03E6b6xtQXX/UyxcCCVKmJ1IRHJDxUJEXMaylzbSfdWTNGYH3/ZZQO3aZicSkdxSsRARl7BnWTz1Xn8QH7LYe0cv6n4+yuxIIpIHKhYiYrrzp61kdX+AMiRwIqAONTd9rsWaIm5KxUJETJWVabC16QgaZW4mySuYkuti8A4qZnYsEckjFQsRMdV3HT/jvoQZ2PAiZdo3BDaobHYkEbkNOt1UREzz+efw6poo7qA+vgP6UvOxDmZHEpHbpGIhIqbYsgWGDYNMKvLji5sY/5qv2ZFExAE0FSIi+S7hFyvvdFlDZiZ07w4vvOqnxZoiHiLXxWL9+vV07dqV8PBwLBYLCxcudEIsEfFUmVaDbU2H893Fe5lY+t98+SV46U8cEY+R61/n9PR06tWrx5QpU5yRR0Q83PyoaXQ9/xk2vOgXXZvAQLMTiYgj5XqNRadOnejUqZMzsoiIh1v07AZ6bXgKgCODJnLHoPYmJxIRR3P64k2r1YrVar32cUpKirNfUkRc0LaYX2n2zoMUJpv9dR+i1owxZkcSESdw+sxmdHQ0QUFB124RERHOfkkRcTFnjmVQqM/9hHGOk0F1qblxhhZringopxeLcePGkZycfO12+vRpZ7+kiLiQjAyY3n4BDbK2keRdgpANC7EUK2p2LBFxEqdPhfj6+uLrq/PTRQoiw4Dhw+Hz4/1IKpLB2I/LE167ktmxRMSJtEGWiDjNxx/n7K7p5QX3xQwmXGs1RTxerqdC0tLSiI2NJTY2FoATJ04QGxvLqVOnHJ1NRNzY5vm/UvLJvpTkAm+9Be1VKkQKBIthGEZuvmDt2rW0adPmT/cPGDCAWbNm/e3Xp6SkEBQURHJyMoE6gV3EI52OyyCx5j00zN7G9rLdaHR6kdZqiri5W33/zvVUSOvWrcllFxGRAuTqFYNdzYfRLTtnsWatFZNVKkQKEG2kKyIOYxjwbespdLs4CxteZMyah39NLdYUKUhULETEYeY/tZ6Htz0NwPEhkwh7JMrkRCKS31QsROS2ZWbC9PGnafVRzs6ahxo9TNWpo82OJSIm0OmmIpJnhgFLlsDTT8OpuBCiCOBqibJUX/epdtYUKaA0YiEieXJwdyaT635G9/uyiYuD4qF+HHhxDhF7l2IpWsTseCJiEo1YiEiuXL4M3w5cSqtFT/M0hznuZaXImCd48UUIDGxmdjwRMZmKhYjckuxsmP/6IUq+OZohWUsBSPIpzYvRxQnTcgoR+Y2KhYj8rZ9+SOLEwFfpe/FDCpNNFoU502sUFWeMB210JyJ/oDUWInJTJ07AAw9AYrdBPHrxPQqTzfE63bAc2E/FbyepVIjIn2jEQkT+JC0NJr5h4533vLFa4Revl7izeBzFpr9L5IO66IeI3JyKhYhcY7fDwvdO4DP+WUpklMfKv2nbFiZPbkB4rT06hVRE/paKhYgAsHV1Gvv6T+Th+Hfww0qGxY87vniBjo+U+q1PqFSIyN/TGguRAu7MaTvTWn5N2bbVGRT/Bn5YOVn5Xry2baVT/1IapBCRXNGIhUgBlZEBs8YdpsEH/2CofTMA5wMiKTT5XSoO7K5pDxHJExULkQLGMOD772HMGLh6Mog49nHFqygXh40n4p1R4OdndkQRcWMqFiIFyJ6tGSwcuIgJB/oAULZsGDsenkerkfWJKBtucjoR8QQqFiIFQOJ5gwWPxNBh5TP8i5NsKVyCxuPa8dxzULRoZ7PjiYgHUbEQ8WBZWTDvxT2U//cohtrWAHDRvywzp2RSeqDJ4UTEI6lYiHioVfMukDj0JfomfYI3djIsfpzr/ywVPn4eihY1O56IeCidbiriYQ4fhi6dDYIfas9DSdPwxs7xxr0pfPQQFb54VaVCRJxKxULEQyQlwTOjDWrXhiVLLbzl9QJnQuqT9uM6IrfNwzuygtkRRaQAULEQcXM2G8x77QhbQruS9t4nZGdDly7w6v4HKHt2O8U632N2RBEpQLTGQsSNbfgxmWMDX6Nv4gf4kEWDQju4//uBdOjqQ84W3N5mRxSRAkYjFiJu6JfjNqY1nkG1+6oyIPFdfMjiRM3OBO9a81upEBExh4qFiBtJT4epj+3gcpXGDN3xGKVJJKF4dZK/WUKl/T9SuHZ1syOKSAGnYiHiBgwDZs+G6tVhxgyoa+wmzTuI+GffI+z8XoIe6mR2RBERQGssRFzejvXpfDV0I+8fbA9A4YqN2NHtSxq/2IFipUNMTicicj2NWIi4qLPxBtNbzaF0qxpMOngfdf3jePNNOHgQmrz/CBaVChFxQRqxEHExGRkwd8x2akwdyRD7zwAkFqvIqq/OUapHVZPTiYj8NRULERdhGLBsVgJpT73Ao2mz8MLgqlcRLjz2AhHvjQZ/f7Mjioj8LRULERewbx88+5SVz9Y0JJyzABy7qz+VvokmIqKsyelERG6d1liImOjiRRg+HOrVg2VrfJnq/SSnw5tyZdUmKm/4Ei+VChFxMyoWIibIyoLZ4/axNyyKgx+vxm6HBx6AQQfGEHF6E0XuvdPsiCIieaKpEJF8tmb+Rc4+PoGHLk/FGzul/FO48OMWWrexAIXNjicicltULETySdzBbFb3nkavff+iDZcBON7gAe749m28q1hMTici4hiaChFxspQUmNp3PZk16zNk35OU4DLxpeqQumg1kTvn412lktkRRUQcRsVCxElsNvjsM6haFdbMTaAW+0kuXJKECVMJP7uTgG5tzI4oIuJwmgoRcYJNy1OYNvIgXx5uBsDuqr040Dqemm8NICg42OR0IiLOoxELEQc6ddLO9GafU6ljNd4+3JVyAcm8+y7s3Weh5iejQKVCRDycRixEHODKFfhmxEYazBrJEGMHAOeCqhK76DQlWwWZnE5EJP+oWIjcBsOAHz4+jf255xl85RsA0rwDSX7qX5Sd+CT4+JicUEQkf6lYiOTRjh0wYdh55m67g2KkY8fCybb/pNLs1ykWWtrseCIiptAaC5FcSkiAwYOhSRP4cVtpYrx7carC3WT+vIPI/36CRaVCRAowFQuRW2S1whcjd3KwXBRrPz+GYUC/ftDm4MeUP7EOv+YNzI4oImI6TYWI/A3DgBVfnSPpyfH0T/kMLwymBb9A0cXzaNECQJczFxH5nYqFyF84EJvJht4f0CfuNYJIAeD4nQ/Tdu5EvCqYHE5ExAVpKkTkBi5dgqndllKoQW0ej3uWIFL4Nawx6Ss3ErlpNl4VIsyOKCLiklQsRP4gOxs+/jhnG+4zP+ygGnFc9g3l/FszKXdmC0WjWpgdUUTEpWkqROQ36xZe5t3nzvFDXA0AltV8hoFNofL7T0FgoMnpRETcg0YspMA7djib6fU+plbPqrwa9xClgm1MmQKbd/tTeeZ4lQoRkVzQiIUUWKmpMPfx1TSfN5Ihxj4AMkuEcWTtWYLrlDM5nYiIe9KIhRQ4djvMn3Scn0Lu57G5balt7COlUDBnX/yI8HOxKhUiIrdBIxZSoGzaBB/8cw+zDjTBl0yy8eZ0l2FUnPUygaVKmh1PRMTtqVhIgfDrrzB2LMyeDVCHp7ybUCbSn/B5k6nUoJbZ8UREPEaepkKmTJlCxYoV8fPzo1mzZmzdutXRuUQc4upV+Pzxzeyp0JX/zE7BYoHBgy1UPvgjFQ+vwEelQkTEoXJdLObNm8fo0aOZMGECO3fupF69enTo0IHz5887I59InhgG/DD9DEtD+jPo0+Z0ti/mo3IT2bYNZsyA0lWDwGIxO6aIiMexGIZh5OYLmjVrRpMmTfjoo48AsNvtRERE8OSTTzJ27Ni//fqUlBSCgoJITk4m0NGn8SUk5JwaWKSIY59X3Erspqts7/sufX+JpihXci5n3noglea8gaVMmNnxRETc0q2+f+dqjUVmZiY7duxg3Lhx1+7z8vIiKiqKTZs23fBrrFYrVqv1umDO0LAhzNnXjhpZ+7hSKIDUImFcDQojq2QYhIVhqVqF7CeeonRpCA4Gy+VLOSWkkJaZeIrz5+H7fgvo+N9n+Ce/AHAqogWl57xPZMvGJqcTESkYcvWueuHCBWw2G6GhodfdHxoayqFDh274NdHR0bzyyit5T3iLjh+HIlnJABTJTqVISiqkxMHpnM/vXVabuh8+BUDhwrDHuIdq2QdI9gkhtWgYGUFhZJXKKSFeVSqTPehxQkOhZEnwzkjPGQXR0LlLysyEjz6CV16BD1MWUZFfuOhfDmPS25Qf3kffNxGRfOT0P9fHjRvH6NGjr32ckpJCRITjL+C0cyecSfiFPSdTSTuaQMbJBGxnErCcS6DwpQTOXg0mKAuSkyErC4JJxAuD4MzzBGeeh8t74GTOc+2hDvUmPw6AlxccsDSlkv0ol33DckpI8TCyS5XJGQmpEont4UcJDYXSpaEwWTnNRfLFf79J5JXxWWw4Hg7AnNrRtGpamQofPqspMRERE+SqWJQqVQpvb2/OnTt33f3nzp0jLOzGc9e+vr74+vrmPeEtioyEyEgLtAgEAoFqf3rMc0BGBiQmwun4ePbEXSD92B9KyPlz+FxK4Iy1FKXscPFizmZKpUjAh0xCM04RmnEKLgLHcp5zD3Vo9O6j117jkFc9yhBPkm8YqcVySoitVBiUCcOrSiT2B3pfKyH+fob+ms6jg7szWf/QFPoceoWRRHGk9Hyio2HAgLJ4e08wO56ISIGVq2Lh4+NDo0aNWLVqFT169AByFm+uWrWKESNGOCOfw/n5QUQERER4Q7NQIBSo96fHjSTnSpeJifDrqTPsP3LuWgmxx+eMhPhcSuDXrFDKeOXM79tsEGJPIJBkAq8mw9XDkAjE5TxnLPVoMKn3tdfY51WX0pZELvuVIa1YGNbiYdhCfp+OicS4ryuhoRAaCsWKqYMAXL4M3w1cQqtFTzOEIwA0L32cuF1pBIYXMzmdiIjkeipk9OjRDBgwgMaNG9O0aVMmT55Meno6AwcOdEY+UxUqBGXKQJkyftCsAlDhho8bSs7IxqVLcO7IMY78VkIyT+WMhHidT8DncgKnsssSURjOnctZF1DGfoYSXCYk/RykA+eAwznPuYv6NJzY9dpr7LQ0oqT3ZZJ9Q0kL+P8SYgkPw7tyJYx27a+VkOBgzyshNht899ohSr45msezlgKQ5FMa64Q3Kfv8P8Db29yAIiIC5KFY9OnTh8TERP71r3+RkJBA/fr1WbZs2Z8WdBY0Xl5QqhSUKhUMLYKBGjd83CBy9lhISYEL+w5w8nDCtRJiP3MW78ScEnLSqECkX04JSU+HisZxgrOTIPtETglJAH5bL7uL+jR8s/2119hiaUZx71SS/cJIDwgjIzgM+28lpFDlilha3XNtOqZUKdd/T16zBr4dvJwPTtxHYbLJojBnHhxJxRnjISjI7HgiIvIHud7H4nY5dR8LD5WeDpe2Hyf5yDmuHE/AeioBIz5nJMT3cgInvSrxUrHJnDsHSUmQRBBB3Pi03p00oBE7r328jnsIKpROsn9OCckMDsNeOgxLmTC8K1ekUPMm10ZCSpfO33WpJ07As8/CggXgzxWOWGqQWas+EfPepXDNqvkXREREnLOPhZijaFEo2iqSiFaRN/x8E6DXb/9ttcKlDZtIOJLA1RO/jYScTcA7MaeEnPauSu3gnJGQCxegvhFLYHYqpJJziwf25zzXDhrSmB3XXmclUQQUupozEhL4WwkJLYNXeBiFK5enUMO610pIaCj4++fteNPSYO7Qtfh/8zkx9pl4eXkzaFgRijy5g3LVQ/L2pCIiki9ULDyMry+UaVuTMm1r3vDzdYHuv/13dpZB0uo1HP+9hJxOgN9LSFIC53zuoH6pnBJy/jw0tW3NKSFp5Nz+UEK20ZimbLv2OovpQhFvKyn+YaQHliGzRM5IiHfZMApHRuBXt9p1IyEBATlTRAsnn6Twi8/yz4z5AMTXaE3HbwdRpw6ASoWIiKvTVIjcErvNIHX5z6T8VkKyTidgnE3A+0ICfkkJHPWvw4thn3PuXE4RSbQGEEDaDZ/rf0vI9/TExyubjMIBdLF+jx9WbHjxS8ehVPryFSwhpfLrMEVE5CY0FSIO5eVtIajzXQR1vvHnKwHtfvtvw26Q/kMMCXH/X0JIyCkh/klnuVK0Gi3Kc62EtEtbSTF7Ovy28/vJyvcSPm8ykY3q5MehiYiIA6lYiMNZvCwU6x7FzXaVCAc2/v6BYWBd8A0X4hLI+OUcQW0aUrF3J887X1ZEpIBQsRBzWSz4PtgV5+/NKiIi+cHL7AAiIiLiOVQsRERExGFULERERMRhVCxERETEYVQsRERExGFULERERMRhVCxERETEYVQsRERExGFULERERMRhVCxERETEYVQsRERExGFULERERMRhVCxERETEYfL96qaGYQCQkpKS3y8tIiIiefT7+/bv7+M3k+/FIjU1FYCIiIj8fmkRERG5TampqQQFBd308xbj76qHg9ntduLj4wkICMBisTjseVNSUoiIiOD06dMEBgY67Hldiacfo47P/Xn6Mer43J+nH6Mzj88wDFJTUwkPD8fL6+YrKfJ9xMLLy4ty5co57fkDAwM98ofljzz9GHV87s/Tj1HH5/48/RiddXx/NVLxOy3eFBEREYdRsRARERGH8Zhi4evry4QJE/D19TU7itN4+jHq+Nyfpx+jjs/9efoxusLx5fviTREREfFcHjNiISIiIuZTsRARERGHUbEQERERh1GxEBEREYfxyGLRrVs3ypcvj5+fH2XKlKF///7Ex8ebHcthTp48yeDBg6lUqRL+/v5UrlyZCRMmkJmZaXY0h3njjTdo0aIFRYoUoXjx4mbHcYgpU6ZQsWJF/Pz8aNasGVu3bjU7ksOsX7+erl27Eh4ejsViYeHChWZHcqjo6GiaNGlCQEAApUuXpkePHhw+fNjsWA4zdepU6tate21TpebNm7N06VKzYznNxIkTsVgsjBo1yuwoDvPyyy9jsViuu9WoUcOULB5ZLNq0acO3337L4cOHWbBgAceOHePBBx80O5bDHDp0CLvdzvTp09m/fz/vvfce06ZN44UXXjA7msNkZmbSq1cvhg0bZnYUh5g3bx6jR49mwoQJ7Ny5k3r16tGhQwfOnz9vdjSHSE9Pp169ekyZMsXsKE6xbt06hg8fzubNm1m5ciVZWVm0b9+e9PR0s6M5RLly5Zg4cSI7duxg+/bt3HvvvXTv3p39+/ebHc3htm3bxvTp06lbt67ZURyuVq1anD179tptw4YN5gQxCoBFixYZFovFyMzMNDuK00yaNMmoVKmS2TEcbubMmUZQUJDZMW5b06ZNjeHDh1/72GazGeHh4UZ0dLSJqZwDMGJiYsyO4VTnz583AGPdunVmR3Ga4OBgY8aMGWbHcKjU1FSjatWqxsqVK41WrVoZI0eONDuSw0yYMMGoV6+e2TEMwzAMjxyx+KNLly4xe/ZsWrRoQeHChc2O4zTJycmUKFHC7BhyA5mZmezYsYOoqKhr93l5eREVFcWmTZtMTCZ5lZycDOCRv3M2m425c+eSnp5O8+bNzY7jUMOHD6dLly7X/S56kri4OMLDw4mMjKRfv36cOnXKlBweWyyef/55ihYtSsmSJTl16hSLFi0yO5LTHD16lA8//JAhQ4aYHUVu4MKFC9hsNkJDQ6+7PzQ0lISEBJNSSV7Z7XZGjRrFXXfdRe3atc2O4zB79+6lWLFi+Pr6MnToUGJiYqhZs6bZsRxm7ty57Ny5k+joaLOjOEWzZs2YNWsWy5YtY+rUqZw4cYK7776b1NTUfM/iNsVi7Nixf1qY8r+3Q4cOXXv8s88+y65du1ixYgXe3t48+uijGC6+yWhujxHgzJkzdOzYkV69evHYY4+ZlPzW5OX4RFzN8OHD2bdvH3PnzjU7ikNVr16d2NhYtmzZwrBhwxgwYAAHDhwwO5ZDnD59mpEjRzJ79mz8/PzMjuMUnTp1olevXtStW5cOHTqwZMkSkpKS+Pbbb/M9i9ts6Z2YmMjFixf/8jGRkZH4+Pj86f5ff/2ViIgIfv75Z5ce2svtMcbHx9O6dWvuvPNOZs2ahZeXa/fEvHwPZ82axahRo0hKSnJyOufJzMykSJEizJ8/nx49ely7f8CAASQlJXncaJrFYiEmJua6Y/UUI0aMYNGiRaxfv55KlSqZHcepoqKiqFy5MtOnTzc7ym1buHAhPXv2xNvb+9p9NpsNi8WCl5cXVqv1us95iiZNmhAVFZXvozSF8vXVbkNISAghISF5+lq73Q6A1Wp1ZCSHy80xnjlzhjZt2tCoUSNmzpzp8qUCbu976M58fHxo1KgRq1atuvZma7fbWbVqFSNGjDA3nNwSwzB48skniYmJYe3atR5fKiDnZ9TV/828VW3btmXv3r3X3Tdw4EBq1KjB888/75GlIi0tjWPHjtG/f/98f223KRa3asuWLWzbto2WLVsSHBzMsWPHeOmll6hcubJLj1bkxpkzZ2jdujUVKlTgnXfeITEx8drnwsLCTEzmOKdOneLSpUucOnUKm81GbGwsAFWqVKFYsWLmhsuD0aNHM2DAABo3bkzTpk2ZPHky6enpDBw40OxoDpGWlsbRo0evfXzixAliY2MpUaIE5cuXNzGZYwwfPpw5c+awaNEiAgICrq2NCQoKwt/f3+R0t2/cuHF06tSJ8uXLk5qaypw5c1i7di3Lly83O5pDBAQE/Gk9zO9r8DxlncyYMWPo2rUrFSpUID4+ngkTJuDt7U3fvn3zP4y5J6U43p49e4w2bdoYJUqUMHx9fY2KFSsaQ4cONX799VezoznMzJkzDeCGN08xYMCAGx7fmjVrzI6WZx9++KFRvnx5w8fHx2jatKmxefNmsyM5zJo1a274/RowYIDZ0RziZr9vM2fONDuaQwwaNMioUKGC4ePjY4SEhBht27Y1VqxYYXYsp/K000379OljlClTxvDx8THKli1r9OnTxzh69KgpWdxmjYWIiIi4PtefmBcRERG3oWIhIiIiDqNiISIiIg6jYiEiIiIOo2IhIiIiDqNiISIiIg6jYiEiIiIOo2IhIiIiDqNiISIiIg6jYiEiIiIOo2IhIiIiDqNiISIiIg7zf/mgVTmV0SDyAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(inputs, gelu_gt, 'b')\n",
    "plt.plot(inputs, gelu_test, 'r--')\n",
    "%%plt.plot(inputs, intgelu_test[0], 'g-.')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aec96baa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hao/.conda/envs/torch/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers.models.ibert.quant_modules import IntGELU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "13698603",
   "metadata": {},
   "outputs": [],
   "source": [
    "intgelu = IntGELU()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e53bfcb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs_int, inputs_scale = tensor_quant_scale(inputs, bits=32, scale=.1)\n",
    "intgelu_test = intgelu(inputs, torch.Tensor([inputs_scale]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "900f03f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ -0.,  -0., -10., -30., -50.])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(intgelu_test[0] / intgelu_test[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "adb2a918",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1\n"
     ]
    }
   ],
   "source": [
    "print(inputs_scale)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dc4faf1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-1.1829])\n"
     ]
    }
   ],
   "source": [
    "print(intgelu_test[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "779157c7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
